#!/usr/bin/env bash
#
# 1.0.2
#   增加对socket的监控
#
# 1.0.3
#   增加收集apm coredump文件的bt信息
#
# 1.0.4
#   增加收集系统时间、iptables状态、SELinux状态
#
# 1.0.5
#   增加收集/var/log/messages*中的信息
#
# 1.0.6
#   增加es的jstack和jmap信息
#   增加--tar-include参数，指定该参数后，除了apmh.log和jstack和jmap日志外，只包含以下参数指定的日志
#       --tar-coredump
#       --tar-broker
#       --tar-es
#       --tar-collect
#       --tar-hub
#       --tar-das
#       --tar-console
#
# 1.0.7
#   调整压缩后的文件名为apmh.logs.$(date +"%Y-%m-%d-%H-%M-%S").tbz2
#
# 1.0.8
#   压缩日志时忽略report_data*.log

VERSION=1.0.8
PROG=`basename $0`

OPT_DEBUG=0
OPT_COLOR=0
OPT_SAVE_LOGS=0
OPT_TAR_LOGS=0
OPT_TAR_TIME=
OPT_NO_CHECK_NETWORK=0
OPT_NO_CHECK_JSTACK=0
OPT_NO_CHECK_JMAP=0
OPT_TAR_INCLUDE=0
OPT_TAR_COREDUMP=0
OPT_TAR_BROKER=0
OPT_TAR_ES=0
OPT_TAR_COLLECT=0
OPT_TAR_HUB=0
OPT_TAR_DAS=0
OPT_TAR_CONSOLE=0

KRED=
KRED_BACK=
KRED_UNDER=
KGREEN=
KGREEN_BACK=
KGREEN_UNDER=
KYELLOW=
KYELLOW_BACK=
KYELLOW_UNDER=
KBLUE=
KBLUE_BACK=
KBLUE_UNDER=
KPINK=
KPINK_BACK=
KPINK_UNDER=
KCYAN=
KCYAN_BACK=
KCYAN_UNDER=
RESET=

APM_DIR=/opt/apm
APM_BROKER_FILE=${APM_DIR}/analyze/dist/bin/apm
APM_ES_FILE=${APM_DIR}/server/bin/apm_es
APM_COLLECT_FILE=${APM_DIR}/server/bin/apm_collect
APM_HUB_FILE=${APM_DIR}/server/bin/apm_hub
APM_DAS_FILE=${APM_DIR}/server/bin/apm_das
APM_CONSOLE_FILE=${APM_DIR}/server/bin/apm_console
APM_JDBC_CONF=${APM_DIR}/server/conf/jdbc.properties
APM_BROKER_BIN_DIR=${APM_DIR}/analyze/dist/bin
APM_BROKER_CLIENT=${APM_BROKER_BIN_DIR}/client

TAR_FILE=${APM_DIR}/apmh.logs.$(date +"%Y-%m-%d-%H-%M-%S").tbz2
LOG_DIR=${APM_DIR}/apmh.logs
SAVE_LOG_FILE=${LOG_DIR}/apmh.log
JSTACK_ES_LOG=${LOG_DIR}/jstack_es.log
JSTACK_COLLECT_LOG=${LOG_DIR}/jstack_collect.log
JSTACK_HUB_LOG=${LOG_DIR}/jstack_hub.log
JSTACK_DAS_LOG=${LOG_DIR}/jstack_das.log
JMAP_ES_LOG=${LOG_DIR}/jmap_es.log
JMAP_COLLECT_LOG=${LOG_DIR}/jmap_collect.log
JMAP_HUB_LOG=${LOG_DIR}/jmap_hub.log
JMAP_DAS_LOG=${LOG_DIR}/jmap_das.log

APM_BROKER_LOGS_DIR=analyze/dist/logs
APM_ES_LOGS_DIR=server/es/logs0
APM_COLLECT_LOGS_DIR=server/indicator/logs
APM_HUB_LOGS_DIR=server/hub/logs
APM_DAS_LOGS_DIR=server/threshold/logs
APM_CONSOLE_LOGS_DIR=server/tomcat/logs

HAS_APM_BROKER=0
HAS_APM_ES=0
HAS_APM_COLLECT=0
HAS_APM_HUB=0
HAS_APM_DAS=0
HAS_APM_CONSOLE=0

MYSQL_IS_RUNNING=0
APM_ES_IS_RUNNING=0
APM_COLLECT_IS_RUNNING=0
APM_HUB_IS_RUNNING=0
APM_DAS_IS_RUNNING=0
APM_CONSOLE_IS_RUNNING=0

MYSQL_CLI_IS_OK=0

# 用来检查系统最小的空闲CPU占比
G_CPU_IDLE_MIN=20

function echo2
{
#    if [[ ${OPT_SAVE_LOGS} -eq 1 ]]; then
#        echo $* >> ${SAVE_LOG_FILE}
#    else
        echo $* | tee -a ${SAVE_LOG_FILE}
#    fi
}

function echo_debug
{
    [[ ${OPT_DEBUG} -eq 0 ]] && return
    echo2 -e $1
}

function echo_stopped
{
    if [[ ${OPT_COLOR} -eq 1 ]]; then
        echo2 -e "\\033[70G[  ${KRED}stopped${RESET}  ]"
    else
        echo2 -e "\t\t[  ${KRED}stopped${RESET}  ]"
    fi
}

function echo_running
{
    if [[ ${OPT_COLOR} -eq 1 ]]; then
        echo2 -e "\\033[70G[  ${KGREEN}running${RESET}  ]"
    else
        echo2 -e "\t\t[  ${KGREEN}running${RESET}  ]"
    fi
}

function echo_check_level
{
    [[ -z ${G_LOOP} ]] && G_LOOP=1
    [[ ${G_LOOP} -gt 1 ]] && echo2
    echo2 -ne "${KBLUE}${G_LOOP}. "
    echo2 -e "$1${RESET}"

    [[ ${OPT_TAR_LOGS} -eq 1 ]] && (
        echo -ne "${KBLUE}${G_LOOP}. "
        echo -e "$1${RESET}"
    )

    G_LOOP=$((G_LOOP+1))
}

function bytes_to_human
{
    b=${1:-0}
    d=''
    s=0
    S=(B {K,M,G,T,E,P,Y,Z}B)
    while ((b > 1024)); do
        d="$(printf ".%02d" $((b % 1024 * 100 / 1024)))"
        b=$((b / 1024))
        let s++
    done
    echo "$b$d${S[$s]}"
}

function file_size
{
    [ ! -e ${1} ] && {
        echo 0
        return
    }

    echo "$(stat -c '%s' ${1})"
}

# 获取文件的时间，输出格式 yyyymmdd-hhmmss
function file_datetime
{
    [ ! -e ${1} ] && {
        echo 0
        return
    }

    t="$(stat -c '%y' ${1})"
    date="$(echo ${t} | awk '{print $1}' | awk -F- '{printf("%s%s%s\n", $1, $2, $3)}')"
    time="$(echo ${t} | awk '{print $2}' | awk -F. '{print $1}' | awk -F: '{printf("%s%s%s\n", $1, $2, $3)}')"
    echo "${date}-${time}"
}

function check_system_info
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check system info"

    echo2 -e "${KCYAN}# datetime:${RESET}"
    date +"%F %T" | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# Memory:${RESET}"
    free -g | tee -a ${SAVE_LOG_FILE}

    local num_of_processor=$(grep "processor" /proc/cpuinfo | wc -l)
    local cpu_of_idle=$(top -n1 | grep Cpu | awk -F id '{print $1}' | awk '{print $5}' | awk -F % '{print $1}')

    if [[ `echo "${cpu_of_idle} ${G_CPU_IDLE_MIN}" | awk '{if($1<$2){print 1}else{print 0}}'` -eq 1 ]];then
        cpu_of_idle="${KYELLOW}${cpu_of_idle}${RESET}"
    fi

    echo2 -e "\n${KCYAN}# CPU${RESET}:"
    echo2 -e "\t${num_of_processor} cores"
    echo2 -e "\t${cpu_of_idle}% idle"

    echo2 -e "\n${KCYAN}# Disk:${RESET}"
    df -h | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# ulimit:${RESET}"
    ulimit -a | grep "core file\|open file" | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# /etc/security/limits.conf:${RESET}"
    cat /etc/security/limits.conf | grep "nofile\|core" | grep -v "#" | tee -a ${SAVE_LOG_FILE}

    echo2 -ne "\n${KCYAN}# hostname:${RESET}\n\t"
    hostname -s | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# /etc/hosts:${RESET}"
    cat /etc/hosts | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# iptables:${RESET}"
    service iptables status | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# SELinux:${RESET}"
    sestatus | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# collect \"kill|task\" info from /var/log/message*:${RESET}"
    for file in `ls -rt /var/log/message* 2>/dev/null`; do
        egrep -i "kill|task" ${file} | tee -a ${SAVE_LOG_FILE}
    done
}

function check_network_throughput
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    [[ ${OPT_NO_CHECK_NETWORK} -eq 1 ]] && return

    echo_check_level "check network throughput (in 3 seconds)"
    sar -n DEV 3 1 | tee -a ${SAVE_LOG_FILE}
}

function check_apm_service_exists
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ -f ${APM_BROKER_FILE} ]] && HAS_APM_BROKER=1 && echo_debug "\tHAS ${KGREEN}APM_BROKER${RESET}"
    [[ -f ${APM_ES_FILE} ]] && HAS_APM_ES=1 && echo_debug "\tHAS ${KGREEN}APM_ES${RESET}"
    [[ -f ${APM_COLLECT_FILE} ]] && HAS_APM_COLLECT=1 && echo_debug "\tHAS ${KGREEN}APM_COLLECT${RESET}"
    [[ -f ${APM_HUB_FILE} ]] && HAS_APM_HUB=1 && echo_debug "\tHAS ${KGREEN}APM_HUB${RESET}"
    [[ -f ${APM_DAS_FILE} ]] && HAS_APM_DAS=1 && echo_debug "\tHAS ${KGREEN}APM_DAS${RESET}"
    [[ -f ${APM_CONSOLE_FILE} ]] && HAS_APM_CONSOLE=1 && echo_debug "\tHAS ${KGREEN}APM_CONSOLE${RESET}"
}

function check_mysql_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    pid_mysqld=$(pidof mysqld)
    if [ -z ${pid_mysqld} ]; then
        echo2 -ne "\tMySQL" && echo_stopped
    else
        echo2 -ne "\tMySQL (pid ${pid_mysqld})" && echo_running
        MYSQL_IS_RUNNING=1
    fi
}

function check_broker_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_BROKER} -eq 0 ]] && return

    pid_apm=$(pidof apm)
    if [ -z ${pid_apm} ]; then
        echo2 -ne "\tUYUN APM Broker apm" && echo_stopped
    else
        echo2 -ne "\tUYUN APM Broker apm(pid ${pid_apm})" && echo_running
    fi

    pid_apmdssl=$(pidof apmdssl)
    if [ -z ${pid_apmdssl} ]; then
        echo2 -ne "\tUYUN APM Broker dssl" && echo_stopped
    else
        echo2 -ne "\tUYUN APM Broker dssl(pid ${pid_apmdssl})" && echo_running
    fi
}

function check_es_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_ES} -eq 0 ]] && return

    pid_es=$(ps -Af | grep java | grep apm | grep Elasticsearch | awk '{print $2}')
    if [ -z ${pid_es} ]; then
        echo2 -ne "\tUYUN APM Elasticsearch" && echo_stopped
    else
        xms_es=$(ps -Af | grep java | grep apm | grep Elasticsearch | grep -P 'Xms\d+[mgMG]' -o)
        xmx_es=$(ps -Af | grep java | grep apm | grep Elasticsearch | grep -P 'Xmx\d+[mgMG]' -o)
        if [[ -n ${xms_es} && ${xmx_es} ]]; then
            xm="(${xms_es} ${xmx_es})"
        elif [[ -n ${xms_es} ]]; then
            xm="(${xms_es}, ${KYELLOW}Xmx is not set${RESET})"
        elif [[ -n ${xmx_es} ]]; then
            xm="(${KYELLOW}Xms is not set${RESET}, ${xmx_es})"
        else
            xm="(${KYELLOW}Xms and Xmx is not set${RESET})"
        fi
        echo2 -ne "\tUYUN APM Elasticsearch (pid ${pid_es})${xm}" && echo_running
        APM_ES_IS_RUNNING=1
    fi
}

function check_collect_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_COLLECT} -eq 0 ]] && return

    pid_collect=$(ps -Af | grep java | grep apm | grep indicator | awk '{print $2}')
    if [ -z ${pid_collect} ]; then
        echo2 -ne "\tUYUN APM Collect" && echo_stopped
    else
        xms_es=$(ps -Af | grep java | grep apm | grep indicator | grep -P 'Xms\d+[mgMG]' -o)
        xmx_es=$(ps -Af | grep java | grep apm | grep indicator | grep -P 'Xmx\d+[mgMG]' -o)
        if [[ -n ${xms_es} && ${xmx_es} ]]; then
            xm="(${xms_es} ${xmx_es})"
        elif [[ -n ${xms_es} ]]; then
            xm="(${xms_es}, ${KYELLOW}Xmx is not set${RESET})"
        elif [[ -n ${xmx_es} ]]; then
            xm="(${KYELLOW}Xms is not set${RESET}, ${xmx_es})"
        else
            xm="(${KYELLOW}Xms and Xmx is not set${RESET})"
        fi
        echo2 -ne "\tUYUN APM Collect (pid ${pid_collect})${xm}" && echo_running
        APM_COLLECT_IS_RUNNING=1
    fi
}

function check_hub_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_HUB} -eq 0 ]] && return

    pid_hub=$(ps -Af | grep java | grep apm | grep "hub/bin" | awk '{print $2}')
    if [ -z ${pid_hub} ]; then
        echo2 -ne "\tUYUN APM Hub" && echo_stopped
    else
        xms_es=$(ps -Af | grep java | grep apm | grep "hub/bin" | grep -P 'Xms\d+[mgMG]' -o)
        xmx_es=$(ps -Af | grep java | grep apm | grep "hub/bin" | grep -P 'Xmx\d+[mgMG]' -o)
        if [[ -n ${xms_es} && ${xmx_es} ]]; then
            xm="(${xms_es} ${xmx_es})"
        elif [[ -n ${xms_es} ]]; then
            xm="(${xms_es}, ${KYELLOW}Xmx is not set${RESET})"
        elif [[ -n ${xmx_es} ]]; then
            xm="(${KYELLOW}Xms is not set${RESET}, ${xmx_es})"
        else
            xm="(${KYELLOW}Xms and Xmx is not set${RESET})"
        fi
        echo2 -ne "\tUYUN APM Hub (pid ${pid_hub})${xm}" && echo_running
        APM_HUB_IS_RUNNING=1
    fi
}

function check_das_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_DAS} -eq 0 ]] && return

    pid_das=$(ps -Af | grep java | grep apm | grep Bootstrap | grep threshold | awk '{print $2}')
    if [ -z ${pid_das} ]; then
        echo2 -ne "\tUYUN APM Das" && echo_stopped
    else
        xms_es=$(ps -Af | grep java | grep apm | grep Bootstrap | grep threshold | grep -P 'Xms\d+[mgMG]' -o)
        xmx_es=$(ps -Af | grep java | grep apm | grep Bootstrap | grep threshold | grep -P 'Xmx\d+[mgMG]' -o)
        if [[ -n ${xms_es} && ${xmx_es} ]]; then
            xm="(${xms_es} ${xmx_es})"
        elif [[ -n ${xms_es} ]]; then
            xm="(${xms_es}, ${KYELLOW}Xmx is not set${RESET})"
        elif [[ -n ${xmx_es} ]]; then
            xm="(${KYELLOW}Xms is not set${RESET}, ${xmx_es})"
        else
            xm="(${KYELLOW}Xms and Xmx is not set${RESET})"
        fi
        echo2 -ne "\tUYUN APM Das (pid ${pid_das})${xm}" && echo_running
        APM_DAS_IS_RUNNING=1
    fi
}

function check_console_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_CONSOLE} -eq 0 ]] && return

    pid_console=$(ps -Af | grep java | grep apm | grep Bootstrap | grep webapps | awk '{print $2}')
    if [ -z ${pid_console} ]; then
        echo2 -ne "\tUYUN APM Console" && echo_stopped
    else
        xms_es=$(ps -Af | grep java | grep apm | grep Bootstrap | grep webapps | grep -P 'Xms\d+[mgMG]' -o)
        xmx_es=$(ps -Af | grep java | grep apm | grep Bootstrap | grep webapps | grep -P 'Xmx\d+[mgMG]' -o)
        if [[ -n ${xms_es} && ${xmx_es} ]]; then
            xm="(${xms_es} ${xmx_es})"
        elif [[ -n ${xms_es} ]]; then
            xm="(${xms_es}, ${KYELLOW}Xmx is not set${RESET})"
        elif [[ -n ${xmx_es} ]]; then
            xm="(${KYELLOW}Xms is not set${RESET}, ${xmx_es})"
        else
            xm="(${KYELLOW}Xms and Xmx is not set${RESET})"
        fi
        echo2 -ne "\tUYUN APM Console (pid ${pid_console})${xm}" && echo_running
        APM_CONSOLE_IS_RUNNING=1
    fi
}

function check_service_status
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check service status"

    check_mysql_status
    check_broker_status
    check_es_status
    check_collect_status
    check_hub_status
    check_das_status
    check_console_status
}

function check_mysql_cli
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    [[ ${MYSQL_IS_RUNNING} -eq 0 ]] && return

    [[ ! -s ${APM_JDBC_CONF} ]] && echo2 -e "\t${KRED}${APM_JDBC_CONF} is not exists${RESET}" && return

    MYSQL_USERNAME=$(cat ${APM_JDBC_CONF} | grep "db.username" | awk -F = '{print $2}' | tr -d '\r')
    MYSQL_PASSWORD=$(cat ${APM_JDBC_CONF} | grep "db.password" | awk -F = '{print $2}' | tr -d '\r')

    [[ -n ${MYSQL_USERNAME} ]] && [[ -n ${MYSQL_PASSWORD} ]] && MYSQL_CLI_IS_OK=1
}

function check_mysql_config
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check mysql sys config"

    [[ ${MYSQL_CLI_IS_OK} -eq 1 ]] && {
        local processlist_count=$(mysql -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} -e "show processlist" 2>/dev/null | grep "Sleep" | wc -l)
        echo2 -e "\tprocesslist count is ${processlist_count}"
    }

    local max_connections_process
    local max_connections_config

    [[ ${MYSQL_CLI_IS_OK} -eq 1 ]] && {
        max_connections_process=$(mysql -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} -e "show variables like 'max_connections'" 2>/dev/null | grep "max_connections" | awk '{print $2}')
    }

    if [[ ! -s /etc/my.cnf ]]; then
        echo2 -e "\t${KRED}/etc/my.cnf is not exists${RESET}"
    else
        max_connections_config=$(cat /etc/my.cnf | grep "max_connections"| awk -F = '{print $2}')
    fi

    if [[ -z ${max_connections_config} ]]; then
        echo2 -e "\t\"max_connections\" in process is ${max_connections_process}"
        echo2 -e "\t${KRED}\"max_connections\" in /etc/my.cnf is unset${RESET}"
    elif [[ ${max_connections_process} -ne ${max_connections_config} ]]; then
        echo2 -e "\t${KRED}\"max_connections\" in /etc/my.cnf[${max_connections_config}] is not equal to \"max_connections\" in process[${max_connections_process}]${RESET}"
    else
        echo2 -e "\t\"max_connections\" in process is ${KBLUE}${max_connections_process}${RESET}"
        echo2 -e "\t\"max_connections\" in /etc/my.cnf is ${KBLUE}${max_connections_config}${RESET}"
    fi
}

# 检查MySQL数据
function check_mysql_data
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check mysql apm data"

    [[ ${MYSQL_CLI_IS_OK} -eq 1 ]] && {
        local count=$(mysql -Bs -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} apm -e "select count(1) from ip_location" 2>/dev/null)
        if [[ ${count} == "0" ]]; then
            count="${KRED}${count}${RESET}"
        else
            count="${KGREEN}${count}${RESET}"
        fi
        echo2 -e "ip_location count: ${count}"

        local count=$(mysql -Bs -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} apm -e "select count(1) from app_resources" 2>/dev/null)
        echo2 -e "\napp_resources count: ${count}"

        echo2 -e "\ndict_errors:"
        mysql -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} apm -e "select protocol,count(1) from dict_errors group by protocol" 2>/dev/null | tee -a ${SAVE_LOG_FILE}
    }
}

function check_broker_runtime_info
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_BROKER} -eq 0 ]] && return

    echo_check_level "check broker runtime info"

    ${APM_BROKER_CLIENT} -t2 -c "print system;print memory;print module config;" | tee -a ${SAVE_LOG_FILE}
}

# 检查是否存在apm的coredump文件
function check_broker_coredump_files
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"
    [[ ${HAS_APM_BROKER} -eq 0 ]] && return

    echo_check_level "check broker coredump files"

    which gdb >/dev/null 2>&1 || return

    local GDB_CMD=/tmp/apm_gdb.cmd
    echo -e "bt\nq" > ${GDB_CMD}

    rm -rf ${LOG_DIR}/coredump.logs

    local exists_core=0

    cd ${APM_BROKER_BIN_DIR}

    for core_file in `ls core.* 2>/dev/null`; do
        mkdir -p ${LOG_DIR}/coredump.logs
        local t="$(file_datetime ${core_file})"
        echo "gdb apm ${core_file} -x ${GDB_CMD} > ${LOG_DIR}/coredump.logs/${t}-${core_file}.log"
        gdb apm ${core_file} -x ${GDB_CMD} > ${LOG_DIR}/coredump.logs/${t}-${core_file}.log
        exists_core=1
    done

    if [[ ${exists_core} -eq 1 ]]; then
        echo2 "Exists apm coredump files"
    fi

    rm -f ${GDB_CMD} >/dev/null 2>&1
}

function check_es_health
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check es health"

    local es_version=$(curl -s "http://localhost:9200" | grep "number" | grep -P "\d+\.\d+\.\d+" -o)
    if [[ ${es_version} == "2.3.0" ]]; then
        local es_max_file_descriptors=$(curl -s "http://localhost:9200/_nodes/stats/process?pretty" | grep max_file_descriptors | grep -P "\d+" -o)
    else
        local es_max_file_descriptors=$(curl -s "http://localhost:9200/_nodes/process?pretty" | grep max_file_descriptors | grep -P "\d+" -o)
    fi
    local es_cluster_health=$(curl -s -XGET "http://127.0.0.1:9200/_cluster/health?pretty" | grep "status" | grep -P "\w+" -o | tail -n 1)
    if [[ ${es_cluster_health} == "green" ]]; then
        es_cluster_health="${KGREEN}${es_cluster_health}${RESET}"
    elif [[ ${es_cluster_health} == "yellow" ]]; then
        es_cluster_health="${KYELLOW}${es_cluster_health}${RESET}"
    elif [[ ${es_cluster_health} == "red" ]]; then
        es_cluster_health="${KRED}${es_cluster_health}${RESET}"
    fi

    echo2 -e "\tes version: ${es_version}"
    echo2 -e "\tes max file descriptors: ${es_max_file_descriptors}"
    echo2 -e "\tes cluster health: ${es_cluster_health}"
}

# 检查apm、apmdssl、es、collect、hub、das、console的内存占用情况
function check_process_cpu_memory
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check process cpu, memory"

    if [[ -n ${pid_apm} ]]; then
        local mem=`pmap -x ${pid_apm} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Broker apm:\t${mem}G"
    fi

    if [[ -n ${pid_apmdssl} ]]; then
        local mem=`pmap -x ${pid_apmdssl} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Broker apmdssl:\t${mem}G"
    fi

    if [[ -n ${pid_es} ]]; then
        local mem=`pmap -x ${pid_es} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Elasticsearch:\t${mem}G"
    fi

    if [[ -n ${pid_collect} ]]; then
        local mem=`pmap -x ${pid_collect} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Collect:\t${mem}G"
    fi

    if [[ -n ${pid_hub} ]]; then
        local mem=`pmap -x ${pid_hub} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Hub:\t\t${mem}G"
    fi

    if [[ -n ${pid_das} ]]; then
        local mem=`pmap -x ${pid_das} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Das:\t\t${mem}G"
    fi

    if [[ -n ${pid_console} ]]; then
        local mem=`pmap -x ${pid_console} 2>/dev/null | awk '/total/ { printf ("%.2f\n", $4 / 1024 / 1024) }'`
        echo2 -e "Console:\t${mem}G"
    fi

    top -d3 -n1 -b | grep "Cpu" | tee -a ${SAVE_LOG_FILE}
    echo2 "TOP 10:"
    top -d3 -n1 -b | grep "%CPU" -A 30 | tee -a ${SAVE_LOG_FILE}
}

# 检查5555、8090、9090、9300的端口连接情况
function check_socket_info
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    echo_check_level "check socket info"

    echo2 -e "${KCYAN}# socket of 5555:${RESET}"
    netstat -tnap | grep 5555 | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# socket of 8090:${RESET}"
    netstat -tnap | grep 8090 | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# socket of 9090:${RESET}"
    netstat -tnap | grep 9090 | tee -a ${SAVE_LOG_FILE}

    echo2 -e "\n${KCYAN}# socket of 9300:${RESET}"
    netstat -tnap | grep 9300 | tee -a ${SAVE_LOG_FILE}
}

# 保存collect、hub、das
function save_java_jstack_info
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    [[ ${OPT_NO_CHECK_JSTACK} -eq 1 ]] && return

    echo_check_level "save jstack of es, collect, hub and das to files"

    [[ -n ${pid_es} ]] && {
        echo2 -e "\tsave jstack of es to ${JSTACK_ES_LOG}"
        jstack ${pid_es} > ${JSTACK_ES_LOG} 2>&1
    }

    [[ -n ${pid_collect} ]] && {
        echo2 -e "\tsave jstack of collect to ${JSTACK_COLLECT_LOG}"
        jstack ${pid_collect} > ${JSTACK_COLLECT_LOG} 2>&1
    }

    [[ -n ${pid_hub} ]] && {
        echo2 -e "\tsave jstack of hub to ${JSTACK_HUB_LOG}"
        jstack ${pid_hub} > ${JSTACK_HUB_LOG} 2>&1
    }

    [[ -n ${pid_das} ]] && {
        echo2 -e "\tsave jstack of das to ${JSTACK_DAS_LOG}"
        jstack ${pid_das} > ${JSTACK_DAS_LOG} 2>&1
    }
}

function save_java_jmap_info
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    [[ ${OPT_NO_CHECK_JMAP} -eq 1 ]] && return

    echo_check_level "save jmap of es, collect, hub and das to files"

    [[ -n ${pid_es} ]] && {
        echo2 -e "\tsave jmap of es to ${JMAP_ES_LOG}"
        jmap -heap ${pid_es} > ${JMAP_ES_LOG} 2>&1
    }

    [[ -n ${pid_collect} ]] && {
        echo2 -e "\tsave jmap of collect to ${JMAP_COLLECT_LOG}"
        jmap -heap ${pid_collect} > ${JMAP_COLLECT_LOG} 2>&1
    }

    [[ -n ${pid_hub} ]] && {
        echo2 -e "\tsave jmap of hub to ${JMAP_HUB_LOG}"
        jmap -heap ${pid_hub} > ${JMAP_HUB_LOG} 2>&1
    }

    [[ -n ${pid_das} ]] && {
        echo2 -e "\tsave jmap of das to ${JMAP_DAS_LOG}"
        jmap -heap ${pid_das} > ${JMAP_DAS_LOG} 2>&1
        echo "jmap -histo:live ${pid_das} | grep broada | head" >> ${JMAP_DAS_LOG}
        jmap -histo:live ${pid_das} | grep broada | head >> ${JMAP_DAS_LOG} 2>&1
    }
}

# 打包所有的日志等信息
function tar_logs
{
    echo_debug "\n${KCYAN}${FUNCNAME[0]}...${RESET}"

    [[ ${OPT_TAR_LOGS} -eq 0 ]] && return

    echo_check_level "tar logs to ${TAR_FILE}"
    echo "tar logs to ${TAR_FILE} now ..."

    cd ${APM_DIR}

    [[ ! -e ${APM_BROKER_LOGS_DIR} ]] && APM_BROKER_LOGS_DIR=
    [[ ! -e ${APM_ES_LOGS_DIR} ]] && APM_ES_LOGS_DIR=
    [[ ! -e ${APM_COLLECT_LOGS_DIR} ]] && APM_COLLECT_LOGS_DIR=
    [[ ! -e ${APM_HUB_LOGS_DIR} ]] && APM_HUB_LOGS_DIR=
    [[ ! -e ${APM_DAS_LOGS_DIR} ]] && APM_DAS_LOGS_DIR=
    [[ ! -e ${APM_CONSOLE_LOGS_DIR} ]] && APM_CONSOLE_LOGS_DIR=

    [[ -n ${OPT_TAR_TIME} ]] && {
        OPT_TAR_TIME="--newer-mtime \"${OPT_TAR_TIME}\""
    }

    if [[ ${OPT_TAR_INCLUDE} -eq 1 ]]; then
        local tar_files=$(basename ${LOG_DIR})
        if [[ ${OPT_TAR_BROKER} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_BROKER_LOGS_DIR}"
        fi
        if [[ ${OPT_TAR_ES} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_ES_LOGS_DIR}"
        fi
        if [[ ${OPT_TAR_COLLECT} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_COLLECT_LOGS_DIR}"
        fi
        if [[ ${OPT_TAR_HUB} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_HUB_LOGS_DIR}"
        fi
        if [[ ${OPT_TAR_DAS} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_DAS_LOGS_DIR}"
        fi
        if [[ ${OPT_TAR_CONSOLE} -eq 1 ]]; then
            tar_files="${tar_files} ${APM_CONSOLE_LOGS_DIR}"
        fi

        eval tar cvjf $(basename ${TAR_FILE}) ${tar_files} --warning=no-file-changed ${OPT_TAR_TIME} --exclude="report_data*.log"
    else
        eval tar cvjf $(basename ${TAR_FILE}) $(basename ${LOG_DIR}) ${APM_BROKER_LOGS_DIR} ${APM_ES_LOGS_DIR} ${APM_COLLECT_LOGS_DIR} ${APM_HUB_LOGS_DIR} ${APM_DAS_LOGS_DIR} ${APM_CONSOLE_LOGS_DIR} --warning=no-file-changed ${OPT_TAR_TIME} --exclude="report_data*.log"
    fi


    local size="$(file_size ${TAR_FILE})"
    local size_readable="$(bytes_to_human ${size})"

    echo "has tar logs to ${TAR_FILE} (${size_readable})"
}

function run_main
{
    [[ ${OPT_SAVE_LOGS} -eq 1 ]] && set_color_off

    echo_debug "${KCYAN}${FUNCNAME[0]}...${RESET}"

    rm -f ${TAR_FILE} ${SAVE_LOG_FILE} ${JSTACK_COLLECT_LOG} ${JSTACK_HUB_LOG} ${JSTACK_DAS_LOG} ${JMAP_COLLECT_LOG} ${JMAP_HUB_LOG} ${JMAP_DAS_LOG}

    [[ ! -e ${LOG_DIR} ]] && mkdir -p ${LOG_DIR}

    echo2 -e "${PROG} ${VERSION}"

    check_system_info
    check_network_throughput
    check_apm_service_exists
    check_service_status
    check_mysql_cli
    check_mysql_config
    check_mysql_data
    check_broker_runtime_info
    check_broker_coredump_files
    check_es_health
    check_process_cpu_memory
    check_socket_info
    save_java_jstack_info
    save_java_jmap_info

    tar_logs
}



##### script settings

function script_init
{
    set_color_on
}

function set_color_on
{
    OPT_COLOR=1

    KRED='\x1B[0;31m'
    KRED_BACK='\x1B[7;31m'
    KRED_UNDER='\x1B[4;31m'
    KGREEN='\x1B[0;32m'
    KGREEN_BACK='\x1B[7;32m'
    KGREEN_UNDER='\x1B[4;32m'
    KYELLOW='\x1B[0;33m'
    KYELLOW_BACK='\x1B[7;33m'
    KYELLOW_UNDER='\x1B[4;33m'
    KBLUE='\x1B[1;34m'
    KBLUE_BACK='\x1B[7;34m'
    KBLUE_UNDER='\x1B[4;34m'
    KPINK='\x1B[1;35m'
    KPINK_BACK='\x1B[7;35m'
    KPINK_UNDER='\x1B[4;35m'
    KCYAN='\x1B[0;36m'
    KCYAN_BACK='\x1B[7;36m'
    KCYAN_UNDER='\x1B[4;36m'
    RESET='\x1B[0m'
}

function set_color_off
{
    OPT_COLOR=0

    KRED=
    KRED_BACK=
    KRED_UNDER=
    KGREEN=
    KGREEN_BACK=
    KGREEN_UNDER=
    KYELLOW=
    KYELLOW_BACK=
    KYELLOW_UNDER=
    KBLUE=
    KBLUE_BACK=
    KBLUE_UNDER=
    KPINK=
    KPINK_BACK=
    KPINK_UNDER=
    KCYAN=
    KCYAN_BACK=
    KCYAN_UNDER=
    RESET=
}

function print_usage
{
    echo -e "Version: ${VERSION}"
    echo -e "Usage: ${PROG}"
    echo -e " -V,\\033[7G--version\\033[40GDisplay the version of ${PROG}"
    echo -e " -h,\\033[7G--help\\033[40GPrint this help"
    echo -e " -D,\\033[7G--debug\\033[40GPrint debug info"
    echo -e " -s,\\033[7G--savelogs\\033[40GSave to log file (default: ${SAVE_LOG_FILE})"
    echo -e " -t,\\033[7G--tarlogs\\033[40GTar logs to file (default: ${TAR_FILE})"
    echo -e "\nNo check options:"
    echo -e "\\033[7G--no-check-network\\033[40GDon't check network throughput"
    echo -e "\\033[7G--no-check-jstack\\033[40GDon't check jstack of java process"
    echo -e "\\033[7G--no-check-jmap\\033[40GDon't check jmap of java process"
    echo -e "\nTar options:"
    echo -e "\\033[7G-m\\033[40G--newer-mtime (format examples: \"1 days ago\", \"6 hours ago\")"
    echo -e "\\033[7G--tar-include\\033[40Gtar only include below specified logs"
    echo -e "\\033[9G--tar-coredump\\033[40Gtar include coredump"
    echo -e "\\033[9G--tar-broker\\033[40Gtar include broker"
    echo -e "\\033[9G--tar-es\\033[40Gtar include es"
    echo -e "\\033[9G--tar-collect\\033[40Gtar include collect"
    echo -e "\\033[9G--tar-hub\\033[40Gtar include hub"
    echo -e "\\033[9G--tar-das\\033[40Gtar include das"
    echo -e "\\033[9G--tar-console\\033[40Gtar include console"
#    echo -e " --no-quiet\t\tShow test process messages"
}

script_init





#-u或--unquote 解析后的参数不要用单引号引起来
#-o或--options选项后面接可接受的短选项，如ab:c::，表示可接受的短选项为-a -b -c，其中-a选项不接参数，-b选项后必须接参数，-c选项的参数为可选的
#-l或--long选项后面接可接受的长选项，用逗号分开，冒号的意义同短选项。
#-n选项后接选项解析错误时提示的脚本名字
ARGS=`getopt --options hVDstm: --long help,version,no-color,debug,savelogs,tarlogs,mtime:,\
no-check-network,no-check-jstack,no-check-jmap,tar-include,tar-coredump,tar-broker,tar-es,tar-collect,tar-hub,tar-das,tar-console -n "${PROG}" -- "$@"`
if [ $? != 0 ]; then
    echo
    print_usage
    exit 1
fi

for v in ${ARGS}; do
    if [[ ${v} == -* ]] ; then
        continue
    fi
#    echo "@@@@@@@ $v"
    # 非 "-" 或者 "--" 开头的参数
    case "$v" in
    esac
done


#将规范化后的命令行参数分配至位置参数（$1,$2,...)
eval set -- "${ARGS}"

while true
do
    case "$1" in
        -V|--version)
            echo -e "${PROG} ${VERSION}"
            exit
            ;;
        -h|--help)
            print_usage
            exit
            ;;
        --no-color)
            set_color_off
            shift
            ;;
        -D|--debug)
            OPT_DEBUG=1
            shift
            ;;
        -s|--savelogs)
            OPT_SAVE_LOGS=1
            shift
            ;;
        -t|--tarlogs)
            OPT_SAVE_LOGS=1
            OPT_TAR_LOGS=1
            shift
            ;;
        -m|--mtime)
            case "$2" in
                "") exit ;;
                *)
                    OPT_TAR_TIME="$2"
                    shift 2
                    ;;
            esac
            ;;
        --no-check-network)
            OPT_NO_CHECK_NETWORK=1
            shift
            ;;
        --no-check-jstack)
            OPT_NO_CHECK_JSTACK=1
            shift
            ;;
        --no-check-jmap)
            OPT_NO_CHECK_JMAP=1
            shift
            ;;
        --tar-include)
            OPT_TAR_INCLUDE=1
            shift
            ;;
        --tar-coredump)
            OPT_TAR_COREDUMP=1
            shift
            ;;
        --tar-broker)
            OPT_TAR_BROKER=1
            shift
            ;;
        --tar-es)
            OPT_TAR_ES=1
            shift
            ;;
        --tar-collect)
            OPT_TAR_COLLECT=1
            shift
            ;;
        --tar-hub)
            OPT_TAR_HUB=1
            shift
            ;;
        --tar-das)
            OPT_TAR_DAS=1
            shift
            ;;
        --tar-console)
            OPT_TAR_CONSOLE=1
            shift
            ;;
        --)
            shift
            break
            ;;
        *)
            echo "Internal error!"
            exit 1
            ;;
    esac
done

run_main
